package com.zhiying.crawler.multiThreadClaw;


import com.zhiying.crawler.dao.SortDao;
import com.zhiying.crawler.dto.SortDTO;
import org.apache.http.HttpResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;

import java.util.List;
import java.util.Map;
import java.util.concurrent.*;
import java.util.stream.Collectors;

@Component
public class SortClawer {
    private static final Logger logger = LoggerFactory.getLogger(SortClawer.class);

    @Autowired
    private SortDao sortDao;

    private static String PAGE_INDEX = "https://www.88dus.com/";


    @Scheduled(cron = "0 24 15 * * ?")
    public void getSortsAndBookList(){
        HttpResponse response = ClawUtil.mockLoginBbdus();
        /*暂且先做此判断，后期优化增加登录不成功重试机制*/
        if(response == null){
            logger.error("mock login to get cookie error");
            return;
        }
        List<SortDTO> sortDTOS = ClawUtil.getAllSortLinks(response);
        sortDao.saveSortInfo(sortDTOS);

        //filter出未爬的链接 测试先爬type1的数据
        List<SortDTO> untrackSorts = sortDao.getSortList();
//        List<SortDTO> untrackSorts = sortDTOS.stream().filter(s->(!s.getStatus() && s.getSortType()==1)).collect(Collectors.toList());
        List<String> sortLinks = untrackSorts.stream().map(s->s.getSortLink()).collect(Collectors.toList());

        ScheduledExecutorService ses = Executors.newScheduledThreadPool(40);
        BlockingQueue<Map<String,String>> queue = new LinkedBlockingQueue<>(60);//默认长度是0
        for (final SortDTO sortDTO : untrackSorts) {
            ses.scheduleWithFixedDelay(new Runnable() {
                @Override
                public void run() {
                    try {
                        Map<String, String> html = ClawUtil.accessByCookie2(response, sortDTO.getSortLink());
                        queue.put(html);//当前空间不足，调用此方法的线程被阻断直到queue有空间为止
                    } catch (InterruptedException e) {
                        e.printStackTrace();
                    }
                }
            }, 0, 6, TimeUnit.SECONDS);//间隔6秒爬一次 initialDelay=0的时候，表示不延时启动线程
        }
        for (SortDTO sortDTO : untrackSorts) {
            try {
                Thread.sleep(5000);
                Map<String, String> htmlMap = queue.take();//一开始直接取，疑似产生死锁。take put在互相等待
                for (Map.Entry<String, String> entry : htmlMap.entrySet()) {
                    String climbLink = entry.getKey();
                    String pageHtml = entry.getValue();
                    SortDTO sort = new SortDTO(true,climbLink,pageHtml);
                    sortDao.updateSortInfo(sort);
                }

            } catch (InterruptedException e) {
                e.printStackTrace();
            }

        }
        logger.info("关闭线程池啦！");
        ses.shutdown();//关闭线程池
    }
}



